{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: requests-html in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (0.10.0)\n",
      "Requirement already satisfied: requests in d:\\py\\lib\\site-packages (from requests-html) (2.27.1)\n",
      "Requirement already satisfied: bs4 in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (from requests-html) (0.0.1)\n",
      "Requirement already satisfied: pyquery in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (from requests-html) (2.0.0)\n",
      "Requirement already satisfied: pyppeteer>=0.0.14 in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (from requests-html) (1.0.2)\n",
      "Requirement already satisfied: w3lib in d:\\py\\lib\\site-packages (from requests-html) (1.21.0)\n",
      "Requirement already satisfied: fake-useragent in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (from requests-html) (1.2.1)\n",
      "Requirement already satisfied: parse in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (from requests-html) (1.19.1)\n",
      "Requirement already satisfied: importlib-metadata>=1.4 in d:\\py\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (4.11.3)\n",
      "Requirement already satisfied: certifi>=2021 in d:\\py\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (2021.10.8)\n",
      "Requirement already satisfied: appdirs<2.0.0,>=1.4.3 in d:\\py\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (1.4.4)\n",
      "Requirement already satisfied: websockets<11.0,>=10.0 in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (from pyppeteer>=0.0.14->requests-html) (10.4)\n",
      "Requirement already satisfied: pyee<9.0.0,>=8.1.0 in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (from pyppeteer>=0.0.14->requests-html) (8.2.2)\n",
      "Requirement already satisfied: urllib3<2.0.0,>=1.25.8 in d:\\py\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (1.26.9)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.42.1 in d:\\py\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (4.64.0)\n",
      "Requirement already satisfied: zipp>=0.5 in d:\\py\\lib\\site-packages (from importlib-metadata>=1.4->pyppeteer>=0.0.14->requests-html) (3.7.0)\n",
      "Requirement already satisfied: colorama in d:\\py\\lib\\site-packages (from tqdm<5.0.0,>=4.42.1->pyppeteer>=0.0.14->requests-html) (0.4.4)\n",
      "Requirement already satisfied: beautifulsoup4 in d:\\py\\lib\\site-packages (from bs4->requests-html) (4.11.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in d:\\py\\lib\\site-packages (from beautifulsoup4->bs4->requests-html) (2.3.1)\n",
      "Requirement already satisfied: importlib-resources>=5.0 in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (from fake-useragent->requests-html) (6.0.1)\n",
      "Requirement already satisfied: cssselect>=1.2.0 in c:\\users\\86180\\appdata\\roaming\\python\\python39\\site-packages (from pyquery->requests-html) (1.2.0)\n",
      "Requirement already satisfied: lxml>=2.1 in d:\\py\\lib\\site-packages (from pyquery->requests-html) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in d:\\py\\lib\\site-packages (from requests->requests-html) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in d:\\py\\lib\\site-packages (from requests->requests-html) (3.3)\n",
      "Requirement already satisfied: six>=1.4.1 in d:\\py\\lib\\site-packages (from w3lib->requests-html) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install requests-html\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (liepindata.py, line 44)",
     "output_type": "error",
     "traceback": [
      "Traceback \u001b[1;36m(most recent call last)\u001b[0m:\n",
      "  File \u001b[0;32mD:\\py\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3369\u001b[0m in \u001b[0;35mrun_code\u001b[0m\n    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  Input \u001b[0;32mIn [2]\u001b[0m in \u001b[0;35m<cell line: 2>\u001b[0m\n    import crawl_liepin\n",
      "\u001b[1;36m  File \u001b[1;32m~\\Desktop\\2022-2023\\大三①\\Week13\\qizhong\\week13qizhong\\crawl_liepin.py:1\u001b[1;36m in \u001b[1;35m<module>\u001b[1;36m\u001b[0m\n\u001b[1;33m    import liepindata\u001b[0m\n",
      "\u001b[1;36m  File \u001b[1;32m~\\Desktop\\2022-2023\\大三①\\Week13\\qizhong\\week13qizhong\\liepindata.py:44\u001b[1;36m\u001b[0m\n\u001b[1;33m    def headers(用户登录cookie):\u001b[0m\n\u001b[1;37m    ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "# 猎聘 没有设置登陆cookie，仅可抓取10页\n",
    "import crawl_liepin\n",
    "\n",
    "crawl_liepin.crawl(城市=\"深圳\",关键词=\"广告策划\",学历='本科',工作经验='应届生')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. cookies实现登陆，获取完整数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 建立登陆cookie\n",
    "cookie = \"inited_user=daf7251f92024e8969feb28b0e9ad34c; XSRF-TOKEN=SIVa3Y_gRaqR7k-mI1I6kQ; __gc_id=d474cd1529ae40a29b8f9c81b7e60de7; _ga=GA1.1.1301373590.1698838360; __uuid=1698838362686.73; __tlog=1698838362732.34%7C00000000%7C00000000%7C00000000%7C00000000; acw_tc=276077be16988383924452585e8d8e297846bb9fcc8c8f34695b420d154b08; Hm_lvt_a2647413544f5a04f00da7eee0d5e200=1698838365; UniqueKey=95507c72a8d5ae141a667e00ad0d9493; liepin_login_valid=0; lt_auth=u%2B5bbHQGxlzxtXfR3zQN4vociI39UWvIpX8EhE0Ahoe%2BCqG04PngSwOGq7EExAMhlkh1ccULN7n2Pev2zXtP4kcTwGqnl4CyvOW92GECTeNcN8W2vezHl8zRQpQcl0AC8nFbtkIL%2BQ%3D%3D; access_system=C; user_roles=0; user_photo=5f8fa3a679c7cc70efbf444e08u.png; user_name=%E8%AE%B8%E6%99%BA%E8%B6%85; need_bind_tel=false; new_user=false; c_flag=fa43f4d55f3df63a96a7b4f194e214d4; inited_user=daf7251f92024e8969feb28b0e9ad34c; Hm_lpvt_a2647413544f5a04f00da7eee0d5e200=1698838461; imId=c5f9b89f8466dffe6882ca1e5431db9c; imId_0=c5f9b89f8466dffe6882ca1e5431db9c; imClientId=c5f9b89f8466dffeb1921abcfab3aed0; imClientId_0=c5f9b89f8466dffeb1921abcfab3aed0; imApp_0=1; __session_seq=7; __uv_seq=7; fe_im_socketSequence_new_0=1_1_1; __tlg_event_seq=116; fe_im_opened_pages=; fe_im_connectJson_0=%7B%220_95507c72a8d5ae141a667e00ad0d9493%22%3A%7B%22socketConnect%22%3A%222%22%2C%22connectDomain%22%3A%22liepin.com%22%7D%7D; _ga_54YTJKWN86=GS1.1.1698838360.1.1.1698838528.0.0.0\"\n",
    "cookie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import crawl_liepin\n",
    "\n",
    "广告策划_深圳_results = crawl_liepin.crawl(城市=\"深圳\",关键词=\"广告策划\",工作经验='',登录cookie=cookie)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. 使用数据分析进行数据交互式可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "广告策划_深圳_results.to_html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "广告策划_深圳_results.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_dq = 广告策划_深圳_results['job.dq'].value_counts()\n",
    "series_dq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dq_name= [i for i in series_dq.index if '-' in i]\n",
    "dq_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_dq[dq_name].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install pyecharts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyecharts import options as opts\n",
    "from pyecharts.charts import Map\n",
    "from pyecharts.faker import Faker\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[i.split('-')[1] for i in dq_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_dq[dq_name].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyecharts import options as opts\n",
    "from pyecharts.charts import Map\n",
    "from pyecharts.faker import Faker\n",
    "\n",
    "c = (\n",
    "    Map()\n",
    "    .add(\"广告策划地区分布\", [list(z) for z in zip([i.split('-')[1] for i in dq_name],series_dq[dq_name].values.tolist())], \"深圳\")\n",
    "    .set_global_opts(\n",
    "        title_opts=opts.TitleOpts(title=\"Map-深圳-广告策划\"), visualmap_opts=opts.VisualMapOpts()\n",
    "    )\n",
    "    \n",
    ")\n",
    "c.render_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "url = \"https://www.liepin.com\"\n",
    "\n",
    "df = pd.read_html(liepin.com, encoding=\"utf8\")[0]\n",
    "\n",
    "df.to_excel(\"广告策划表格数据.xlsx\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
